{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scikit-learn MNIST Model Deployment\n",
    "\n",
    " * Wrap a Scikit-learn MNIST python model for use as a prediction microservice in seldon-core\n",
    "   * Run locally on Docker to test\n",
    "   * Deploy on seldon-core running on minikube\n",
    " \n",
    "## Dependencies\n",
    "\n",
    " * [Helm](https://github.com/kubernetes/helm)\n",
    " * [Minikube](https://github.com/kubernetes/minikube)\n",
    " * [S2I](https://github.com/openshift/source-to-image)\n",
    "\n",
    "```bash\n",
    "pip install sklearn\n",
    "pip install seldon-core\n",
    "```\n",
    "\n",
    "## Train locally\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-1-2e859e898107>:8: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "0.9552363636363637\n",
      "Classification report for classifier RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
      "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
      "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "            min_samples_leaf=1, min_samples_split=2,\n",
      "            min_weight_fraction_leaf=0.0, n_estimators=30, n_jobs=None,\n",
      "            oob_score=False, random_state=None, verbose=0,\n",
      "            warm_start=False):\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.98      0.98      2699\n",
      "           1       0.98      0.98      0.98      3121\n",
      "           2       0.94      0.95      0.95      2698\n",
      "           3       0.94      0.94      0.94      2816\n",
      "           4       0.95      0.95      0.95      2649\n",
      "           5       0.94      0.94      0.94      2491\n",
      "           6       0.97      0.98      0.97      2705\n",
      "           7       0.97      0.96      0.96      2889\n",
      "           8       0.94      0.94      0.94      2667\n",
      "           9       0.94      0.93      0.94      2765\n",
      "\n",
      "   micro avg       0.96      0.96      0.96     27500\n",
      "   macro avg       0.95      0.95      0.95     27500\n",
      "weighted avg       0.96      0.96      0.96     27500\n",
      "\n",
      "\n",
      "Confusion matrix:\n",
      "[[2654    0    6    1    3    4   11    0   19    1]\n",
      " [   0 3067   19    5    8    3    2    7    7    3]\n",
      " [  12    6 2571   20   23    5   21   18   20    2]\n",
      " [   0    6   46 2639    1   43    7   28   34   12]\n",
      " [   7    3   13    1 2524    3   14    8   12   64]\n",
      " [  12    6    6   63   10 2337   15    3   24   15]\n",
      " [  12    5    4    2    7   28 2638    0    9    0]\n",
      " [   5   12   33    8   24    0    0 2768    5   34]\n",
      " [   8   14   20   30   15   29   16   11 2494   30]\n",
      " [  13    6   11   25   52   25    2   24   30 2577]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['sk.pkl']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn import datasets, metrics\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.datasets import fetch_mldata\n",
    "from sklearn.externals import joblib\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\")\n",
    "\n",
    "mnist_images = mnist.train.images\n",
    "mnist_labels = mnist.train.labels\n",
    " # To apply a classifier on this data, we need to flatten the image, to\n",
    "    # turn the data in a (samples, feature) matrix:\n",
    "n_samples = len(mnist_images)\n",
    "data = mnist_images.reshape((n_samples, -1))\n",
    "targets = mnist_labels\n",
    "\n",
    "data,targets = shuffle(data,targets)\n",
    "classifier = RandomForestClassifier(n_estimators=30)\n",
    "\n",
    "# We learn the digits on the first half of the digits\n",
    "classifier.fit(data[:n_samples // 2], targets[:n_samples // 2])\n",
    "\n",
    "# Now predict the value of the digit on the second half:\n",
    "expected = targets[n_samples // 2:]\n",
    "test_data = data[n_samples // 2:]\n",
    "\n",
    "print(classifier.score(test_data, expected))\n",
    "\n",
    "predicted = classifier.predict(data[n_samples // 2:])\n",
    "\n",
    "print(\"Classification report for classifier %s:\\n%s\\n\"\n",
    "          % (classifier, metrics.classification_report(expected, predicted)))\n",
    "print(\"Confusion matrix:\\n%s\" % metrics.confusion_matrix(expected, predicted))\n",
    "\n",
    "joblib.dump(classifier, 'sk.pkl')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wrap model using s2i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "---> Installing dependencies ...\n",
      "Looking in links: /whl\n",
      "Collecting scipy>=0.13.3 (from -r requirements.txt (line 1))\n",
      "  Url '/whl' is ignored. It is either a non-existing path or lacks a specific scheme.\n",
      "Downloading https://files.pythonhosted.org/packages/7f/5f/c48860704092933bf1c4c1574a8de1ffd16bf4fde8bab190d747598844b2/scipy-1.2.1-cp36-cp36m-manylinux1_x86_64.whl (24.8MB)\n",
      "Collecting scikit-learn>=0.18 (from -r requirements.txt (line 2))\n",
      "  Url '/whl' is ignored. It is either a non-existing path or lacks a specific scheme.\n",
      "Downloading https://files.pythonhosted.org/packages/5e/82/c0de5839d613b82bddd088599ac0bbfbbbcbd8ca470680658352d2c435bd/scikit_learn-0.20.3-cp36-cp36m-manylinux1_x86_64.whl (5.4MB)\n",
      "Requirement already satisfied: numpy>=1.8.2 in /usr/local/lib/python3.6/site-packages (from scipy>=0.13.3->-r requirements.txt (line 1)) (1.16.2)\n",
      "Installing collected packages: scipy, scikit-learn\n",
      "Successfully installed scikit-learn-0.20.3 scipy-1.2.1\n",
      "Url '/whl' is ignored. It is either a non-existing path or lacks a specific scheme.\n",
      "You are using pip version 19.0.3, however version 19.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!s2i build . seldonio/seldon-core-s2i-python3:0.13 sk-mnist:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a8adfe1e38ed793348c8889a3eb7db82602d77da14b4a9c81cb69105cc32fbf8\r\n"
     ]
    }
   ],
   "source": [
    "!docker run --name \"mnist_predictor\" -d --rm -p 5000:5000 sk-mnist:0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Send some random features that conform to the contract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "SENDING NEW REQUEST:\n",
      "\n",
      "[[0.312 0.488 0.971 0.609 0.807 0.484 0.306 0.549 0.293 0.224 0.394 0.242\n",
      "  0.447 0.501 0.642 0.975 0.627 0.775 0.575 0.062 0.206 0.249 0.848 0.126\n",
      "  0.022 0.435 0.441 0.292 0.968 0.456 0.288 0.703 0.026 0.064 0.159 0.782\n",
      "  0.098 0.33  0.8   0.023 0.642 0.678 0.637 0.151 0.588 0.257 0.812 0.262\n",
      "  0.9   0.978 0.531 0.769 0.042 0.595 0.042 0.335 0.521 0.427 0.246 0.167\n",
      "  0.602 0.081 0.845 0.664 0.98  0.215 0.331 0.439 0.754 0.649 0.525 0.685\n",
      "  0.964 0.771 0.16  0.787 0.23  0.533 0.349 0.909 0.016 0.16  0.522 0.632\n",
      "  0.804 0.091 0.871 0.36  0.563 0.027 0.154 0.963 0.221 0.224 0.836 0.504\n",
      "  0.547 0.624 0.192 0.456 0.238 0.129 0.755 0.132 0.638 0.191 0.393 0.377\n",
      "  0.985 0.425 0.634 0.577 0.567 0.446 0.083 0.673 0.757 0.834 0.93  0.32\n",
      "  0.435 0.34  0.927 0.91  0.533 0.937 0.458 0.463 0.077 0.652 0.516 0.113\n",
      "  0.946 0.952 0.314 0.744 0.125 0.751 0.92  0.322 0.404 0.279 0.258 0.979\n",
      "  0.268 0.443 0.773 0.032 0.397 0.871 0.554 0.664 0.969 0.212 0.272 0.535\n",
      "  0.417 0.756 0.733 0.545 0.818 0.081 0.406 0.268 0.47  0.151 0.316 0.161\n",
      "  0.524 0.156 0.279 0.705 0.008 0.559 0.438 0.353 0.744 0.707 0.901 0.593\n",
      "  0.25  0.303 0.947 0.927 0.811 0.752 0.936 0.748 0.229 0.989 0.33  0.705\n",
      "  0.374 0.931 0.636 0.593 0.218 0.141 0.038 0.892 0.406 0.511 0.145 0.902\n",
      "  0.82  0.391 0.556 0.245 0.315 0.734 0.691 0.62  0.339 0.744 0.523 0.445\n",
      "  0.798 0.685 0.617 0.045 0.32  0.694 0.074 0.02  0.726 0.574 0.845 0.436\n",
      "  0.133 0.842 0.545 0.473 0.626 0.332 0.214 0.892 0.354 0.889 0.789 0.805\n",
      "  0.015 0.591 0.044 0.678 0.178 0.39  0.849 0.474 0.752 0.979 0.018 0.167\n",
      "  0.604 0.092 0.805 0.475 0.627 0.851 0.371 0.41  0.439 0.588 0.83  0.753\n",
      "  0.312 0.622 0.21  0.653 0.134 0.7   0.767 0.853 0.837 0.777 0.86  0.565\n",
      "  0.997 0.296 0.339 0.511 0.616 0.008 0.655 0.641 0.984 0.267 0.068 0.949\n",
      "  0.822 0.55  0.915 0.048 0.154 0.096 0.318 0.806 0.99  0.136 0.912 0.165\n",
      "  0.154 0.451 0.792 0.302 0.596 0.908 0.384 0.317 0.305 0.18  0.437 0.759\n",
      "  0.655 0.211 0.384 0.938 0.949 0.18  0.133 0.315 0.048 0.996 0.538 0.677\n",
      "  0.333 0.149 0.941 0.469 0.795 0.624 0.682 0.328 0.539 0.383 0.01  0.514\n",
      "  0.996 0.088 0.893 0.819 0.13  0.567 0.903 0.018 0.285 0.687 0.321 0.189\n",
      "  0.032 0.712 0.778 0.902 0.741 0.628 0.937 0.344 0.51  0.112 0.463 0.855\n",
      "  0.709 0.491 0.675 0.819 0.123 0.688 0.895 0.808 0.646 0.072 0.88  0.256\n",
      "  0.675 0.628 0.582 0.638 0.125 0.367 0.186 0.624 0.358 0.389 0.254 0.482\n",
      "  0.444 0.842 0.206 0.417 0.595 0.334 0.758 0.357 0.537 0.65  0.447 0.44\n",
      "  0.777 0.454 0.947 0.05  0.544 0.106 0.722 0.269 0.633 0.832 0.372 0.115\n",
      "  0.853 0.834 0.157 0.272 0.623 0.785 0.962 0.664 0.853 0.571 0.969 0.567\n",
      "  0.865 0.171 0.416 0.415 0.509 0.805 0.072 0.546 0.627 0.513 0.606 0.992\n",
      "  0.061 0.045 0.59  0.149 0.773 0.949 0.94  0.838 0.366 0.666 0.964 0.49\n",
      "  0.343 0.543 0.679 0.327 0.071 0.138 0.315 0.367 0.378 0.297 0.474 0.545\n",
      "  0.625 0.106 0.049 0.377 0.04  0.245 0.18  0.263 0.389 0.378 0.646 0.322\n",
      "  0.975 0.778 0.19  0.163 0.086 0.864 0.363 0.322 0.612 0.484 0.931 0.684\n",
      "  0.645 0.287 0.249 0.511 0.189 0.735 0.655 0.734 0.322 0.712 0.128 0.014\n",
      "  0.556 0.39  0.101 0.365 0.179 0.252 0.103 0.719 0.966 0.639 0.761 0.141\n",
      "  0.491 0.441 0.663 0.572 0.893 0.959 0.205 0.317 0.093 0.558 0.372 0.312\n",
      "  0.483 0.784 0.766 0.966 0.832 0.055 0.396 0.857 0.332 0.705 0.629 0.174\n",
      "  0.101 0.189 0.82  0.43  0.853 0.365 0.963 0.448 0.184 0.217 0.318 0.681\n",
      "  0.298 0.992 0.205 0.741 0.628 0.207 0.474 0.95  0.684 0.981 0.601 0.719\n",
      "  0.127 0.929 0.395 0.821 0.792 0.978 0.859 0.841 0.533 0.397 0.327 0.645\n",
      "  0.631 0.184 0.972 0.817 0.863 0.637 0.373 0.739 0.992 0.131 0.591 0.259\n",
      "  0.859 0.954 0.281 0.075 0.147 0.064 0.928 0.109 0.593 0.654 0.836 0.038\n",
      "  0.141 0.089 0.048 0.64  0.975 0.72  0.61  0.418 0.174 0.412 0.028 0.863\n",
      "  0.213 0.572 0.162 0.426 0.711 0.26  0.425 0.425 0.355 0.759 0.908 0.352\n",
      "  0.706 0.675 0.607 0.36  0.822 0.705 0.542 0.624 0.961 0.316 0.549 0.382\n",
      "  0.516 0.233 0.359 0.799 0.231 0.069 0.786 0.808 0.189 0.612 0.573 0.458\n",
      "  0.817 0.636 0.478 0.233 0.104 0.924 0.129 0.141 0.233 0.124 0.392 0.426\n",
      "  0.772 0.841 0.4   0.45  0.033 0.523 0.206 0.529 0.761 0.899 0.346 0.626\n",
      "  0.69  0.782 0.714 0.735 0.458 0.769 0.611 0.743 0.226 0.922 0.682 0.709\n",
      "  0.128 0.998 0.309 0.952 0.085 0.299 0.178 0.483 0.31  0.375 0.977 0.13\n",
      "  0.689 0.509 0.875 0.323 0.685 0.853 0.714 0.132 0.283 0.054 0.184 0.472\n",
      "  0.893 0.645 0.838 0.032 0.824 0.327 0.632 0.709 0.105 0.767 0.327 0.782\n",
      "  0.577 0.912 0.589 0.561 0.627 0.703 0.242 0.709 0.281 0.564 0.803 0.918\n",
      "  0.537 0.292 0.311 0.468 0.794 0.083 0.745 0.278 0.907 0.689 0.706 0.833\n",
      "  0.192 0.474 0.2   0.713 0.748 0.66  0.744 0.981 0.879 0.809 0.238 0.091\n",
      "  0.752 0.203 0.328 0.176 0.698 0.858 0.435 0.91  0.982 0.588 0.486 0.774\n",
      "  0.746 0.529 0.213 0.894 0.95  0.377 0.161 0.929 0.806 0.261 0.645 0.065\n",
      "  0.041 0.01  0.86  0.091 0.372 0.653 0.682 0.93  0.774 0.029 0.679 0.518\n",
      "  0.438 0.321 0.893 0.926]]\n",
      "RECEIVED RESPONSE:\n",
      "meta {\n",
      "}\n",
      "data {\n",
      "  names: \"class:0\"\n",
      "  names: \"class:1\"\n",
      "  names: \"class:2\"\n",
      "  names: \"class:3\"\n",
      "  names: \"class:4\"\n",
      "  names: \"class:5\"\n",
      "  names: \"class:6\"\n",
      "  names: \"class:7\"\n",
      "  names: \"class:8\"\n",
      "  names: \"class:9\"\n",
      "  ndarray {\n",
      "    values {\n",
      "      list_value {\n",
      "        values {\n",
      "          number_value: 0.03333333333333333\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.0\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.2\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.16666666666666666\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.1\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.13333333333333333\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.06666666666666667\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.0\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.26666666666666666\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.03333333333333333\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!seldon-core-tester contract.json 0.0.0.0 5000 -p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mnist_predictor\r\n"
     ]
    }
   ],
   "source": [
    "!docker rm mnist_predictor --force"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test using Minikube\n",
    "\n",
    "**Due to a [minikube/s2i issue](https://github.com/SeldonIO/seldon-core/issues/253) you will need [s2i >= 1.1.13](https://github.com/openshift/source-to-image/releases/tag/v1.1.13)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "😄  minikube v0.34.1 on linux (amd64)\n",
      "🔥  Creating virtualbox VM (CPUs=2, Memory=4096MB, Disk=20000MB) ...\n",
      "📶  \"minikube\" IP address is 192.168.99.100\n",
      "🐳  Configuring Docker as the container runtime ...\n",
      "✨  Preparing Kubernetes environment ...\n",
      "🚜  Pulling images required by Kubernetes v1.13.3 ...\n",
      "🚀  Launching Kubernetes v1.13.3 using kubeadm ... \n",
      "🔑  Configuring cluster permissions ...\n",
      "🤔  Verifying component health .....\n",
      "💗  kubectl is now configured to use \"minikube\"\n",
      "🏄  Done! Thank you for using minikube!\n"
     ]
    }
   ],
   "source": [
    "!minikube start --memory 4096"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clusterrolebinding.rbac.authorization.k8s.io/kube-system-cluster-admin created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "$HELM_HOME has been configured at /home/clive/.helm.\n",
      "\n",
      "Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.\n",
      "\n",
      "Please note: by default, Tiller is deployed with an insecure 'allow unauthenticated users' policy.\n",
      "To prevent this, run `helm init` with the --tiller-tls-verify flag.\n",
      "For more information on securing your installation see: https://docs.helm.sh/using_helm/#securing-your-helm-installation\n",
      "Happy Helming!\n"
     ]
    }
   ],
   "source": [
    "!helm init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"tiller-deploy\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"tiller-deploy\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/tiller-deploy -n kube-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   seldon-core\n",
      "LAST DEPLOYED: Thu Apr 25 09:09:18 2019\n",
      "NAMESPACE: seldon-system\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/Secret\n",
      "NAME                                   TYPE    DATA  AGE\n",
      "seldon-operator-webhook-server-secret  Opaque  0     1s\n",
      "\n",
      "==> v1beta1/CustomResourceDefinition\n",
      "NAME                                         AGE\n",
      "seldondeployments.machinelearning.seldon.io  1s\n",
      "\n",
      "==> v1/ClusterRole\n",
      "seldon-operator-manager-role  1s\n",
      "\n",
      "==> v1/ClusterRoleBinding\n",
      "NAME                                 AGE\n",
      "seldon-operator-manager-rolebinding  1s\n",
      "\n",
      "==> v1/Service\n",
      "NAME                                        TYPE       CLUSTER-IP      EXTERNAL-IP  PORT(S)  AGE\n",
      "seldon-operator-controller-manager-service  ClusterIP  10.108.139.240  <none>       443/TCP  1s\n",
      "\n",
      "==> v1/StatefulSet\n",
      "NAME                                DESIRED  CURRENT  AGE\n",
      "seldon-operator-controller-manager  1        1        1s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                                  READY  STATUS             RESTARTS  AGE\n",
      "seldon-operator-controller-manager-0  0/1    ContainerCreating  0         1s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "NOTES: TODO\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install ../../../helm-charts/seldon-core-operator --name seldon-core --set usageMetrics.enabled=true   --namespace seldon-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "partitioned roll out complete: 1 new pods have been updated...\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/seldon-controller-manager -n seldon-system"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Ingress\n",
    "Please note: There are reported gRPC issues with ambassador (see https://github.com/SeldonIO/seldon-core/issues/473)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   ambassador\n",
      "LAST DEPLOYED: Thu Apr 25 09:09:56 2019\n",
      "NAMESPACE: default\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/Service\n",
      "NAME               TYPE          CLUSTER-IP     EXTERNAL-IP  PORT(S)                     AGE\n",
      "ambassador-admins  ClusterIP     10.102.85.86   <none>       8877/TCP                    0s\n",
      "ambassador         LoadBalancer  10.98.255.212  <pending>    80:31305/TCP,443:32008/TCP  0s\n",
      "\n",
      "==> v1/Deployment\n",
      "NAME        DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE\n",
      "ambassador  3        3        3           0          0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                         READY  STATUS             RESTARTS  AGE\n",
      "ambassador-5b89d44544-947t9  0/1    ContainerCreating  0         0s\n",
      "ambassador-5b89d44544-bqqbp  0/1    ContainerCreating  0         0s\n",
      "ambassador-5b89d44544-nj5rf  0/1    ContainerCreating  0         0s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME        SECRETS  AGE\n",
      "ambassador  1        0s\n",
      "\n",
      "==> v1beta1/ClusterRole\n",
      "NAME        AGE\n",
      "ambassador  0s\n",
      "\n",
      "==> v1beta1/ClusterRoleBinding\n",
      "NAME        AGE\n",
      "ambassador  0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "Congratuations! You've successfully installed Ambassador.\n",
      "\n",
      "For help, visit our Slack at https://d6e.co/slack or view the documentation online at https://www.getambassador.io.\n",
      "\n",
      "To get the IP address of Ambassador, run the following commands:\n",
      "NOTE: It may take a few minutes for the LoadBalancer IP to be available.\n",
      "     You can watch the status of by running 'kubectl get svc -w  --namespace default ambassador'\n",
      "\n",
      "  On GKE/Azure:\n",
      "  export SERVICE_IP=$(kubectl get svc --namespace default ambassador -o jsonpath='{.status.loadBalancer.ingress[0].ip}')\n",
      "\n",
      "  On AWS:\n",
      "  export SERVICE_IP=$(kubectl get svc --namespace default ambassador -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\n",
      "\n",
      "  echo http://$SERVICE_IP:\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install stable/ambassador --name ambassador --set crds.keep=false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"ambassador\" rollout to finish: 0 of 3 updated replicas are available...\n",
      "Waiting for deployment \"ambassador\" rollout to finish: 1 of 3 updated replicas are available...\n",
      "Waiting for deployment \"ambassador\" rollout to finish: 2 of 3 updated replicas are available...\n",
      "deployment \"ambassador\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deployment.apps/ambassador"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wrap Model and Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "---> Installing dependencies ...\n",
      "Looking in links: /whl\n",
      "Collecting scipy>=0.13.3 (from -r requirements.txt (line 1))\n",
      "  Url '/whl' is ignored. It is either a non-existing path or lacks a specific scheme.\n",
      "Downloading https://files.pythonhosted.org/packages/7f/5f/c48860704092933bf1c4c1574a8de1ffd16bf4fde8bab190d747598844b2/scipy-1.2.1-cp36-cp36m-manylinux1_x86_64.whl (24.8MB)\n",
      "Collecting scikit-learn>=0.18 (from -r requirements.txt (line 2))\n",
      "  Url '/whl' is ignored. It is either a non-existing path or lacks a specific scheme.\n",
      "Downloading https://files.pythonhosted.org/packages/5e/82/c0de5839d613b82bddd088599ac0bbfbbbcbd8ca470680658352d2c435bd/scikit_learn-0.20.3-cp36-cp36m-manylinux1_x86_64.whl (5.4MB)\n",
      "Requirement already satisfied: numpy>=1.8.2 in /usr/local/lib/python3.6/site-packages (from scipy>=0.13.3->-r requirements.txt (line 1)) (1.16.2)\n",
      "Installing collected packages: scipy, scikit-learn\n",
      "Successfully installed scikit-learn-0.20.3 scipy-1.2.1\n",
      "Url '/whl' is ignored. It is either a non-existing path or lacks a specific scheme.\n",
      "You are using pip version 19.0.3, however version 19.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!eval $(minikube docker-env) && s2i build . seldonio/seldon-core-s2i-python3:0.13 sk-mnist:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io/sk-mnist created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create -f sk_mnist.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"sk-mnist-single-model-3812de6\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"sk-mnist-single-model-3812de6\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/sk-mnist-single-model-3812de6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "SENDING NEW REQUEST:\n",
      "\n",
      "[[0.445 0.858 0.358 0.545 0.91  0.803 0.061 0.822 0.634 0.225 0.17  0.938\n",
      "  0.754 0.539 0.075 0.548 0.34  0.083 0.858 0.835 0.755 0.041 0.058 0.414\n",
      "  0.631 0.897 0.989 0.486 0.62  0.366 0.596 0.74  0.667 0.347 0.923 0.992\n",
      "  0.54  0.88  0.521 0.877 0.945 0.727 0.697 0.543 0.714 0.147 0.361 0.21\n",
      "  1.    0.742 0.666 0.86  0.59  0.607 0.146 0.345 0.342 0.751 0.958 0.869\n",
      "  0.122 0.779 0.468 0.29  0.79  0.857 0.501 0.032 0.466 0.678 0.823 0.268\n",
      "  0.598 0.27  0.55  0.648 0.818 0.732 0.344 0.123 0.735 0.741 0.675 0.68\n",
      "  0.777 0.584 0.104 0.72  0.237 0.237 0.465 0.625 0.705 0.09  0.673 0.422\n",
      "  0.51  0.603 0.271 0.957 0.565 1.    0.441 0.05  0.772 0.4   0.306 0.186\n",
      "  0.822 0.103 0.432 0.151 0.232 0.134 0.163 0.826 0.01  0.756 0.898 0.918\n",
      "  0.783 0.185 0.696 0.194 0.595 0.668 0.869 0.165 0.541 0.287 0.52  0.215\n",
      "  0.007 0.564 0.643 0.809 0.307 0.973 0.671 0.552 0.073 0.843 0.472 0.514\n",
      "  0.2   0.606 0.387 0.23  0.749 0.161 0.898 0.109 0.96  0.984 0.98  0.343\n",
      "  0.999 0.519 0.228 0.095 0.497 0.964 0.627 0.837 0.924 0.545 0.389 0.717\n",
      "  0.185 0.029 0.782 0.75  0.655 0.728 0.338 0.086 0.658 0.722 0.753 0.916\n",
      "  0.245 0.222 0.371 0.974 0.113 0.458 0.285 0.54  0.231 0.418 0.572 0.167\n",
      "  0.682 0.492 0.32  0.675 0.023 0.139 0.789 0.063 0.606 0.721 0.616 0.049\n",
      "  0.011 0.321 0.463 0.015 0.847 0.781 0.725 0.85  0.509 0.453 0.368 0.584\n",
      "  0.499 0.439 0.31  0.964 0.17  0.691 0.721 0.449 0.295 0.796 0.63  0.868\n",
      "  0.816 0.454 0.072 0.318 0.338 0.327 0.238 0.992 0.685 0.508 0.392 0.531\n",
      "  0.644 0.749 0.976 0.319 0.614 0.719 0.838 0.413 0.368 0.834 0.603 0.102\n",
      "  0.949 0.112 0.543 0.558 0.923 0.849 0.168 0.081 0.886 0.569 0.071 0.205\n",
      "  0.355 0.692 0.106 0.155 0.812 0.186 0.474 0.254 0.474 0.997 0.464 0.934\n",
      "  0.04  0.208 0.063 0.472 0.395 0.008 0.42  0.763 0.912 0.537 0.517 0.613\n",
      "  0.262 0.199 0.987 0.56  0.049 0.696 0.973 0.363 0.596 0.526 0.051 0.629\n",
      "  0.773 0.465 0.404 0.496 0.995 0.902 0.582 0.542 0.909 0.153 0.972 0.932\n",
      "  0.05  0.981 0.717 0.433 0.565 0.352 0.135 0.87  0.299 0.611 0.282 0.577\n",
      "  0.427 0.006 0.893 0.35  0.225 0.908 0.008 0.81  0.193 0.249 0.152 0.656\n",
      "  0.736 0.574 0.175 0.294 0.584 0.637 0.479 0.308 0.855 0.61  0.158 0.408\n",
      "  0.332 0.215 0.037 0.571 0.105 0.618 0.119 0.656 0.722 0.056 0.843 0.799\n",
      "  0.381 0.35  0.982 0.06  0.515 0.505 0.549 0.205 0.765 0.704 0.878 0.926\n",
      "  0.445 0.377 0.777 0.541 0.934 0.342 0.26  0.186 0.052 0.821 0.709 0.639\n",
      "  0.717 0.274 0.583 0.792 0.384 0.121 0.062 0.678 0.85  0.719 0.529 0.7\n",
      "  0.36  0.322 0.579 0.719 0.102 0.209 0.742 0.741 0.282 0.697 0.496 0.495\n",
      "  0.961 0.66  0.953 0.287 0.915 0.424 0.145 0.34  0.001 0.598 0.144 0.159\n",
      "  0.074 0.842 0.38  0.552 0.117 0.268 0.917 0.093 0.405 0.967 0.293 0.333\n",
      "  0.204 0.155 0.859 0.733 0.555 0.06  0.666 0.761 0.684 0.628 0.853 0.955\n",
      "  0.721 0.25  0.84  0.877 0.537 0.867 0.816 0.987 0.279 0.233 0.079 0.194\n",
      "  0.794 0.49  0.799 0.346 0.268 0.56  0.085 0.424 0.741 0.588 0.147 0.455\n",
      "  0.099 0.708 0.652 0.123 0.429 0.424 0.128 0.055 0.708 0.608 0.015 0.059\n",
      "  0.392 0.27  0.778 0.472 0.603 0.047 0.666 0.716 0.312 0.101 0.514 0.068\n",
      "  0.241 0.507 0.246 0.955 0.483 0.415 0.084 0.566 0.827 0.688 0.18  0.467\n",
      "  0.917 0.423 0.774 0.473 0.482 0.35  0.536 0.601 0.328 0.561 0.223 0.048\n",
      "  0.706 0.292 0.994 0.084 0.697 0.455 0.54  0.08  0.627 0.648 0.393 0.629\n",
      "  0.413 0.919 0.312 0.836 0.478 0.081 0.774 0.872 0.731 0.299 0.537 0.79\n",
      "  0.203 0.092 0.726 0.503 0.674 0.343 0.643 0.41  0.06  0.431 0.95  0.447\n",
      "  0.805 0.069 0.556 0.829 0.222 0.483 0.963 0.568 0.796 0.877 0.648 0.754\n",
      "  0.971 0.569 0.982 0.897 0.909 0.225 0.394 0.884 0.903 0.683 0.813 0.503\n",
      "  0.199 0.702 0.938 0.493 0.291 0.856 0.612 0.397 0.851 0.057 0.694 0.977\n",
      "  0.532 0.603 0.935 0.331 0.334 0.63  0.042 0.956 0.674 0.658 0.034 0.039\n",
      "  0.789 0.355 0.226 0.306 0.522 0.317 0.184 0.086 0.866 0.199 0.457 0.588\n",
      "  0.224 0.202 0.128 0.737 0.393 0.604 0.774 0.897 0.999 0.224 0.109 0.545\n",
      "  0.154 0.345 0.846 0.975 0.411 0.425 0.218 0.137 0.993 0.039 0.122 0.345\n",
      "  0.852 0.988 0.139 0.124 0.417 0.971 0.523 0.607 0.307 0.822 0.037 0.059\n",
      "  0.111 0.311 0.538 0.551 0.733 0.896 0.759 0.743 0.089 0.515 0.429 0.046\n",
      "  0.81  0.509 0.378 0.683 0.629 0.002 0.395 0.815 0.166 0.343 0.26  0.504\n",
      "  0.812 0.613 0.614 0.019 0.887 0.868 0.482 0.27  0.194 0.902 0.841 0.834\n",
      "  0.34  0.219 0.839 0.155 0.468 0.817 0.852 0.142 0.573 0.477 0.166 0.231\n",
      "  0.672 0.285 0.363 0.497 0.466 0.346 0.207 0.559 0.516 0.333 0.771 0.103\n",
      "  0.877 0.68  0.164 0.693 0.993 0.825 0.617 0.977 0.326 0.353 0.247 0.921\n",
      "  0.682 0.384 0.785 0.044 0.539 0.625 0.338 0.852 0.705 0.948 0.945 0.186\n",
      "  0.438 0.794 0.418 0.927 0.224 0.733 0.953 0.347 0.479 0.276 0.487 0.862\n",
      "  0.513 0.063 0.586 0.021 0.442 0.087 0.685 0.382 0.298 0.266 0.07  0.491\n",
      "  0.499 0.597 0.818 0.593 0.285 0.075 0.706 0.434 0.715 0.945 0.43  0.325\n",
      "  0.555 0.69  0.154 0.994 0.193 0.46  0.597 0.515 0.124 0.654 0.948 0.392\n",
      "  0.871 0.828 0.61  0.173]]\n",
      "RECEIVED RESPONSE:\n",
      "meta {\n",
      "  puid: \"hi6i0bs0mtg5683dossjejbbbn\"\n",
      "  requestPath {\n",
      "    key: \"classifier\"\n",
      "    value: \"sk-mnist:0.1\"\n",
      "  }\n",
      "}\n",
      "data {\n",
      "  names: \"class:0\"\n",
      "  names: \"class:1\"\n",
      "  names: \"class:2\"\n",
      "  names: \"class:3\"\n",
      "  names: \"class:4\"\n",
      "  names: \"class:5\"\n",
      "  names: \"class:6\"\n",
      "  names: \"class:7\"\n",
      "  names: \"class:8\"\n",
      "  names: \"class:9\"\n",
      "  ndarray {\n",
      "    values {\n",
      "      list_value {\n",
      "        values {\n",
      "          number_value: 0.06666666666666667\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.0\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.16666666666666666\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.13333333333333333\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.1\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.06666666666666667\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.1\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.03333333333333333\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.23333333333333334\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.1\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!seldon-core-api-tester contract.json `minikube ip` `kubectl get svc ambassador -o jsonpath='{.spec.ports[0].nodePort}'` \\\n",
    "    sk-mnist --namespace default -p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🔥  Deleting \"minikube\" from virtualbox ...\n",
      "💔  The \"minikube\" cluster has been deleted.\n"
     ]
    }
   ],
   "source": [
    "!minikube delete"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
